﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using HtmlAgilityPack;
using System.Net;
using Googler.Utilities.Http;

namespace HodhodNews.Extraction
{
    public static class HtmlNodeExtensions
    {

        static string[] tagsToSkip = new string[] { "marquee", "a", "style" };
        static string[] attributesToSkip = new string[] { "copyright" };

        public static bool IsEmpty(this HtmlNode node)
        {
            bool isEmpty = node.ChildNodes.Count == 0 && node.InnerText != null && node.InnerText.Trim().Length == 0;
            return isEmpty;
        }
        public static bool IsCommentContainer(this HtmlNode node)
        {
            if (node.Attributes != null)
            {
                var classAttribute = node.Attributes["class"];
                var idAttribute = node.Attributes["id"];
                var nameAttribute = node.Attributes["name"];

                var attributes = new HtmlAttribute[]{
                    classAttribute,
                    idAttribute,
                    nameAttribute
                };

                foreach (var attr in attributes)
                {
                    if (attr == null)
                        continue;

                    if (!string.IsNullOrWhiteSpace(attr.Value))
                    {
                        if (attr.Value.ToLower().Contains("comment"))
                        {
                            return true;
                        }
                    }
                }
            }
            return false;
        }
        public static bool Skip(this HtmlNode node)
        {
            if (tagsToSkip.Contains(node.Name.ToLower()))
                return true;

            if (node.Attributes != null && node.Attributes.Count > 0)
            {
                foreach (var attr in node.Attributes)
                {
                    if (attributesToSkip.Contains(attr.Value.ToLower()))
                        return true;
                }
            }

            return node.IsCommentContainer();
        }
        public static bool CanHoldArtilce(this HtmlNode node)
        {
            if (node.Name != "#text")
                return false;
            if (node.Name == "script" || node.Name == "form" || node.Name == "body")
                return false;
            if (node.InnerHtml.Length == 0 || node.InnerText.Trim().Length <= Paramaters.ArticleLength)
                return false;
            if (node.ChildNodes.Count > 15) // most probably this is a side menu
                return false;


            HtmlNode parentNode = node.ParentNode;

            if (parentNode != null)
            {
                if (parentNode.Name == "select")
                    return false;
                if (parentNode.Name == "script" || parentNode.Name == "form" || parentNode.Name == "body")
                    return false;
                if (node.Name == "#text" & (parentNode.Name == "table" | parentNode.Name == "tr"))
                    return false;
                if (parentNode.Name == "li")
                    return false;
                if (parentNode.ParentNode != null && parentNode.ParentNode.Name == "li")
                    return false;
                if (parentNode.ParentNode.ParentNode != null && parentNode.ParentNode.ParentNode.Name == "li")
                    return false;
            }
            return true;
        }
    }
    public class CommonParentNewsExtractionProvider : NewsExtractionProvider
    {
        int _NodeOrder = 0;
        const string textTag = "#text";
        Dictionary<Guid, ArtilceNode> LeafList;
        string[] articleTopParents = new string[] { "div", "table" };

        private string[] ArabicHumanText { get; set; }
        private string[] EnglishHumanText { get; set; }

        public HtmlDocument Document { get; private set; }
        public List<ArtilceNode> ArticleNodes { get; set; }
        public string Summary { get; private set; }
        public string SeekParagraph { get; set; }
        public ExtracationOptions Options { get; set; }




        int GetNextNodeOrder()
        {
            return ++_NodeOrder;
        }
        private static int GetNumberOfOccurance(HtmlNode node, string tag)
        {
            HtmlNodeCollection nodes = node.SelectNodes(tag);
            return nodes == null ? 0 : nodes.Count;
        }
        string Encode(string text)
        {
            string encodedString = "";
            foreach (char c in text)
            {
                encodedString += "&#" + Convert.ToInt32(c) + ";";
            }
            return encodedString;
        }
        public bool ContainsParagraph(HtmlNode node, string keywords)
        {
            bool contains = node.InnerText.Contains(keywords) || Encode(node.InnerText).Contains(Encode(keywords));
            return contains;
        }
        private bool ContainsHumanText(HtmlNode wn, out Language language)
        {
            foreach (string s in this.ArabicHumanText)
            {
                if (s == null)
                    continue;

                bool yes = wn.InnerText.Contains(s) || Encode(wn.InnerText).Contains(Encode(s));
                if (yes)
                {
                    language = Language.Arabic;
                    return true;
                }
            }

            foreach (string s in this.EnglishHumanText)
            {
                if (s == null)
                    continue;

                bool yes = wn.InnerText.Contains(s) || Encode(wn.InnerText).Contains(Encode(s));
                if (yes)
                {
                    language = Language.English;
                    return true;
                }
            }

            language = Language.Arabic;
            return false;
        }
        private ArtilceNode ConvertToArticleNode(HtmlNode node)
        {
            ArtilceNode stat = new ArtilceNode()
            {
                Ignore = true
            };

            stat.NodeName = node.Name;
            stat.Links = GetNumberOfOccurance(node, "a");
            stat.Images = GetNumberOfOccurance(node, "img");
            stat.ConfidenceLevel = 1;
            stat.OrderIndex = GetNextNodeOrder();
            stat.ArticleTopParent = GetArticleTopParent(node);
            stat.CharactersLength = node.InnerText.Trim().Length;

            if (stat.Images / stat.CharactersLength <= Paramaters.ImagesToCharactersPercentage & stat.Links / stat.CharactersLength <= Paramaters.LinksToCharactersPercentage)
            {
                Language language;
                if (ContainsHumanText(node, out language))
                {
                    stat.Ignore = false;
                    stat.HtmlNode = node;
                    stat.Language = language;
                    if (!string.IsNullOrWhiteSpace(SeekParagraph) && ContainsParagraph(node, this.SeekParagraph))
                    {
                        stat.ConfidenceLevel += 10;
                    }
                }
            }
            return stat;
        }
        private HtmlNode GetArticleTopParent(HtmlNode node)
        {
            HtmlNode nd = node.ParentNode;
            while (nd.ParentNode != null && !articleTopParents.Contains(nd.Name))
            {
                #region commented
                //if (node.ParentNode.Name == "body")
                //{
                //    // strange bug, keep this if statement untill investigating
                //    // http://news.naseej.com/Detail.asp?InSectionID=182&InNewsItemID=386564
                //    return node.ParentNode;
                //} 
                #endregion
                nd = nd.ParentNode;

            }

            return nd;
        }



        /// <summary>
        /// Converts provided html node to article node then add it to artilce nodes or update its confedince level if it was already added before
        /// </summary>
        /// <param name="currentNode">Node to append to artilce nodes</param>
        /// <returns>Artilce node that has been added</returns>
        private ArtilceNode AppendToArticleNodes(HtmlNode currentNode)
        {
            ArtilceNode artilceNode = ConvertToArticleNode(currentNode);

            if (artilceNode.Ignore)
            {
                return artilceNode;
            }

            ArticleNodes.Add(artilceNode);
            HtmlNode parent = GetArticleTopParent(artilceNode.HtmlNode);

            if (LeafList.ContainsKey(parent.OId))
            {
                var currentStatistics = LeafList[parent.OId];
                currentStatistics.ArticleTopParent = parent;
                currentStatistics.ConfidenceLevel++;
                currentStatistics.CharactersLength += artilceNode.CharactersLength;

                currentStatistics.ChildArtilceNodes.Add(artilceNode);
            }
            else
            {
                artilceNode.ArticleTopParent = parent;
                LeafList.Add(parent.OId, artilceNode);
            }
            return artilceNode;
        }

        ExtractedNews ValidateThenExtract(HtmlNode node, int nodeRedundancy)
        {
            //if (nodeRedundancy >= 3)
            //{
            if (IsNewsArticle(node))
            {
                var noiseCancellator = new NoiseCancellator();
                noiseCancellator.CancelNoise(node, this.Options);
                return new ExtractedNews()
                {
                    Text = node.OuterHtml
                };
            }
            //}
            return null;
        }


        public override ExtractedNews Extract(ExtracationOptions options)
        {
            this.Options = options;
            ExtractedNews extractedNews = null;
            Initialize(options);

            var bodyNode = Document.DocumentNode.SelectNodes("//body")[0];

            ExtractArticleNodes(bodyNode);//initalize LeafList and statisticsList

            var list = LeafList.Values.ToList();
            var comments = CommentDetectorProvider.Instance.Detect(new CommentDetectorOptions()
            {
                Source = list
            });
            var redundantLeaf = GetMostPotential(list);

            if (redundantLeaf == null)
            {
                return null;
            }
            extractedNews = ValidateThenExtract(GetArticleTopParent(redundantLeaf.HtmlNode), redundantLeaf.ConfidenceLevel);
            if (extractedNews != null)
            {
                extractedNews.Language = redundantLeaf.Language;
                return extractedNews;
            }

            List<ArtilceNode> parents;
            var redundantParent = GetMostRedundat2LevelParent(LeafList.Values.ToList(), out parents);
            extractedNews = ValidateThenExtract(redundantParent.HtmlNode, redundantParent.ConfidenceLevel);
            if (extractedNews != null)
            {
                extractedNews.Language = redundantParent.Language;
                return extractedNews;
            }

            redundantParent = GetMostRedundat1LevelParent(parents, redundantParent);
            extractedNews = ValidateThenExtract(redundantParent.HtmlNode, redundantParent.ConfidenceLevel);
            if (extractedNews != null)
            {
                extractedNews.Language = redundantParent.Language;
                return extractedNews;
            }


            return null;
        }
        /// <summary>
        /// Extract nodes which contains artilce or piece of artilce
        /// </summary>
        /// <param name="node">Html node to extract from</param>
        /// <returns>returns 'NoExtraChild' or '' or 'Artilce node has been detected, node id ={GUID}'</returns>
        private string ExtractArticleNodes(HtmlNode node)
        {

            //string ret = "-" + node.Name;
            if (node.ChildNodes.Count == 0)
                return "NoExtraChild";

            if (node.Skip())
                return string.Format("'{0}' Node Skiped", node.Name);

            string message = "";
            foreach (HtmlNode currentNode in node.ChildNodes)
            {
                if (currentNode.IsEmpty())
                    continue;

                string result = ExtractArticleNodes(currentNode);

                if (result == "NoExtraChild" && currentNode.CanHoldArtilce())
                {
                    var artilceNode = AppendToArticleNodes(currentNode);
                    if (!artilceNode.Ignore)
                    {
                        message = "Artilce node has been detected, node id =" + artilceNode.HtmlNode.OId;
                    }
                }
            }
            return message;
        }
        private ArtilceNode GetMostRedundatParent(List<ArtilceNode> list)
        {
            ArtilceNode ps = null;
            Dictionary<Guid, ArtilceNode> parents = new Dictionary<Guid, ArtilceNode>();
            foreach (ArtilceNode stat in list)
            {
                HtmlNode parent = GetArticleTopParent(stat.HtmlNode);
                if (parents.ContainsKey(parent.OId))
                    ps = parents[parent.OId];
                else
                {
                    ps = new ArtilceNode();
                    ps.HtmlNode = parent;
                    parents.Add(ps.HtmlNode.OId, ps);
                }
                ps.ConfidenceLevel++;
            }
            return GetMostPotential(parents.Values.ToList());
        }

        private void Initialize(ExtracationOptions options)
        {
            this.Summary = options.NewsSummaryText;
            this.SeekParagraph = this.Summary.TrySubstractWords(0, 5);

            this.ArabicHumanText = new string[]
            {
                    "ا",
                    "أ",
                    "و",
                    "ي",
                    "ى"
            };
            this.EnglishHumanText = new string[]{
                    "a",
                    "e",
                    "i",
                    "o",
                    "y" 
            };

            ArticleNodes = new List<ArtilceNode>();
            LeafList = new Dictionary<Guid, ArtilceNode>();
            HttpDownloader downloader = new HttpDownloader(options.NewsUrl, null, null);
            Document = new HtmlDocument();
            Document.LoadHtml(downloader.GetPage());


        }

        private ArtilceNode GetMostRedundat2LevelParent(List<ArtilceNode> list, out List<ArtilceNode> parentsList)
        {
            var parents = new Dictionary<Guid, ArtilceNode>();
            ArtilceNode ps = null;
            ArtilceNode redundantParent = new ArtilceNode()
            {
                ConfidenceLevel = 1,
                HtmlNode = GetArticleTopParent(GetArticleTopParent(ArticleNodes[0].HtmlNode))
            };
            parents.Add(redundantParent.HtmlNode.OId, redundantParent);

            foreach (ArtilceNode stat in LeafList.Values)
            {
                HtmlNode parent = GetArticleTopParent(GetArticleTopParent(stat.HtmlNode));
                if (parents.ContainsKey(parent.OId))
                {
                    ps = parents[parent.OId];
                }
                else
                {
                    ps = new ArtilceNode();
                    ps.HtmlNode = parent;
                    parents.Add(ps.HtmlNode.OId, ps);
                }
                ps.ConfidenceLevel++;
            }

            redundantParent = GetMostPotential(parents.Values.ToList());
            parentsList = parents.Values.ToList();

            return redundantParent;
        }
        private ArtilceNode GetMostRedundat1LevelParent(List<ArtilceNode> list, ArtilceNode redundantParent)
        {
            var parents = new Dictionary<Guid, ArtilceNode>();
            ArtilceNode ps = null;

            foreach (ArtilceNode stat in LeafList.Values)
            {
                HtmlNode parent = GetArticleTopParent(stat.HtmlNode);
                if (parents.ContainsKey(parent.OId))
                {
                    ps = parents[parent.OId];
                }
                else
                {
                    ps = new ArtilceNode();
                    ps.HtmlNode = parent;
                    parents.Add(ps.HtmlNode.OId, ps);
                }
                ps.ConfidenceLevel++;
            }

            redundantParent = GetMostRedundant(parents.Values.ToList(), redundantParent);
            return redundantParent;
        }


        /// <summary>
        /// Select the items with the highest count of text childs then selects the item with the longest character length
        /// </summary>
        /// <param name="list">List of node statistics</param>
        /// <returns>The node with the most potential to contain the article</returns>
        ArtilceNode GetMostPotential(List<ArtilceNode> list)
        {
            if (list.Count == 0)
            {
                return null;
            }

            list.Sort((s, s2) => s.OrderIndex.CompareTo(s2.OrderIndex));

            var redundantLeaf = list.First();
            int highestCount = 0;
            int howManyItemHasHighestCount = 0;

            foreach (ArtilceNode stat in list)
            {
                if (redundantLeaf.ConfidenceLevel <= stat.ConfidenceLevel)
                {
                    redundantLeaf = stat;
                    if (highestCount == redundantLeaf.ConfidenceLevel)
                    {
                        howManyItemHasHighestCount++;
                    }
                    else
                    {
                        highestCount = redundantLeaf.ConfidenceLevel;
                        howManyItemHasHighestCount = 1;
                    }
                }
            }

            if (howManyItemHasHighestCount > 1)
            {
                var filteredList = list.Where(s => s.ConfidenceLevel == highestCount).ToList();
                filteredList.Sort((s, s1) => s.CharactersLength.CompareTo(s1.CharactersLength));

                var targetLeaf = filteredList.LastOrDefault();
                return targetLeaf;
            }

            return redundantLeaf;
        }
        ArtilceNode GetMostRedundant(List<ArtilceNode> list, ArtilceNode compareWith)
        {
            if (list.Count == 0)
            {
                return null;
            }

            var redundantLeaf = compareWith;

            foreach (ArtilceNode stat in list)
            {
                if (redundantLeaf.ConfidenceLevel < stat.ConfidenceLevel)
                    redundantLeaf = stat;
            }
            return redundantLeaf;
        }
        private bool IsNewsArticle(HtmlNode node)
        {
            bool isNewsArticle = false;

            int charactersLength = node.InnerText.Length;


            if (1.0 * charactersLength / charactersLength < 0.5)
                return isNewsArticle;

            int linksCount = GetNumberOfOccurance(node, "a");
            int ImagesCount = GetNumberOfOccurance(node, "img");

            if (charactersLength > Paramaters.ArticleLength &
                ImagesCount / charactersLength <= Paramaters.ImagesToCharactersPercentage &
                linksCount / charactersLength <= Paramaters.LinksToCharactersPercentage)
            {
                isNewsArticle = true;
                //stat.InnerText = node.InnerText;
            }
            return isNewsArticle;
        }
    }
}
